#coding=utf-8
from __future__ import with_statement
import thread
import os, sys, threading, time, cPickle as pc, Queue
import index, filesystem as fs, vglobals
from logger import *
import EncodingConvert as ec

__doc__ = '''
module info : desktop crawl,
              main indexing thread,
              fs watcher thread
              index queue method
'''

DUMPED_LIST = 'index/indexlist.0'

# desktop crawl thread
class crawl_thread(threading.Thread):
    # crawl over the fs and add files to index
    def run(self):

        #Create index queue if not compeleted
        self.create_index_queue(interval=.2)
        vglobals.LIST_CREATED = True

        

    # create index queue
    def create_index_queue(self,interval):
        if os.path.exists('index/finishedindexing.0'):
           # Creation has already finished.
           print 'index had finished.'
           f2 = open(DUMPED_LIST,'rb')
           vglobals.INDEXLIST = pc.load(f2)
           f2.close()
           for k in vglobals.INDEXLIST:
               vglobals.INDEXQUEUE.put(k,False)
           return

        #init globals
        if vglobals.INDEXLIST == [] or vglobals.INDEXQUEUE == None:
            try:
                f2 = open(DUMPED_LIST,'rb')
                vglobals.INDEXLIST = pc.load(f2)
                f2.close()
                for k in vglobals.INDEXLIST:
                    vglobals.INDEXQUEUE.put(k,False)
                return
            except Exception,rt:
                print 'error',Exception,rt
                vglobals.INDEXQUEUE = Queue.Queue()
                vglobals.INDEXLIST = []

        # get all files in the special dirs
        for dir2index in vglobals.DIRS:
            for root, bdirs, files in os.walk(dir2index):
                # if in exclude list
                for exd in vglobals.EXCLUDE_DIRS:
                       if exd in root: 
                           # not in this dir
                           print 'not in this dir:',root
                           continue
                for f in files:
                    # already consumed by indexer thread
                    if f not in vglobals.INDEXLIST:
                        while vglobals.PAUSE_INDEX == True:
                                time.sleep(.5)
                        time.sleep(vglobals.WALK_INTERVAL)
                        try:
                            # append to index queue
                            fn = str(root)+'\\'+f
                            vglobals.INDEXQUEUE.put(fn,False)
                            vglobals.INDEXLIST.append(fn)
                        except Exception, inst:
                            print '....',Exception, inst
                            continue

        #vglobals.INDEXQUEUE.join()
        log(__name__,'Index queue created.')
        # Create a file to indicate finishing of queue creation
        #open('index/finishedindexing.0','w').close()
        dumpall( None,vglobals.INDEXLIST )
        return

# fs watcher thread
class watcher_thread(threading.Thread):
        def run(self):
            #fs.watch_all_drivers(callback=do_file_change)
            fs.watch_dirs(vglobals.WATCH_DIRS,callback=do_file_change)
            while True:
                time.sleep(.5)
           
def do_file_change(state,filename):
    # exclusion
    for d in vglobals.EXCLUDE_DIRS:
        if d in filename:
            return  
    print 'file :',filename,'has been changed.'
    vglobals.INDEXLIST.append(filename)
    vglobals.INDEXQUEUE.put(filename)



# indexing thread
class indexer_thread(threading.Thread):
    # index the queue items if exists
    def run(self):
        import index,database
        index.db = database.database()
        # always at work
        while True:
            time.sleep(vglobals.INDEX_INTERVAL)
            if vglobals.STOP_FLAG == True:
                print __name__,': I am back.'
                return
            while vglobals.PAUSE_INDEX == True:
                time.sleep(.5)
            # wait until index list created
            if  vglobals.LIST_CREATED == False: 
                time.sleep(1)
                continue
            if not vglobals.INDEXQUEUE.empty() or len(vglobals.INDEXLIST) == 0:
                file = vglobals.INDEXQUEUE.get()
                #vglobals.INDEXLIST.remove(vglobals.INDEXLIST.index(file))
                try:
                    log(__name__,'Start to index '+file+'...')
                    index.add_file(file)   
                    log(__name__,'File '+file+' added.')
                    vglobals.INDEXLIST.remove(file)
                except  Exception, inst:
                    log(__name__,'inst:'+str(Exception)+str(inst)+ec.zh2gbk(file)[1],level=LVL_DEBUG)
                    continue
            

# dump list and queue into disk
def dumpall(queue,lst):
    f1 = open('index/indexlist.0','wb')
    #f2 = open('index/indexqueue.0','w')
    pc.dump(lst,f1,-1) 
    #pc.dump(queue,f2)
    f1.close()
    #f2.close()

if __name__=='__main__':
    # tests
    f = crawl_thread().start()
